
/*
 * ArffTableModel.java
 *
 * Created on 2007-nov-10, 22:11:27
 *
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 *///TODO Init FoldColumn after all data is added;
package grex;

import grex.genes.GeneException;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Serializable;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.Vector;

import javax.swing.table.DefaultTableModel;

/**
 *
 * @author RIK
 */
public class ArffTableModel extends DefaultTableModel implements Serializable {

    protected ArrayList<ArrayList<String>> arlAllowedValues = new ArrayList<ArrayList<String>>();
    protected ArrayList<ArrayList<Double>> arlNumValues = new ArrayList<ArrayList<Double>>();
    protected ArrayList<Hashtable<String, Integer>> arlCatValues = new ArrayList<Hashtable<String, Integer>>(); //Used to count the number of accurneces of each clategory
    private String tableName;
    private Boolean[] missingValues;
    protected ArrayList<String> arlNumericTypes = new ArrayList<String>();
    private Hashtable<String, Integer> hstFoldCounters = new Hashtable<String, Integer>();
    private static String FOLD_COLUMN_NAME = "foldColumn";
    protected int foldColumnIndex = -1, targetColumnIndex = -1, nrOfFolds = 10, foldSeed = 1;
    private boolean hasFoldSpecification, orginalFoldSpec = false;
    private String[] columnNames;
    private Hashtable<String, double[][]> hstTrainFolds;
    private Hashtable<String, double[][]> hstValFolds;
    private Hashtable<String, double[][]> hstTestFolds;
    private Hashtable<String, double[]> hstTrainFoldMinValues;
    private Hashtable<String, double[]> hstTrainFoldMaxValues;

    private Hashtable<String, ArrayList<ArrayList<Double>>> hstFoldSplits;
    private ArrayList<double[]> arlNumData, arlNormNumData;
    private ArrayList<String> arlFileComments = new ArrayList<String>();
    private ArrayList<Integer> arlCategoryColumnIndexes = new ArrayList<Integer>();
    private ArrayList<Integer> arlContinousColumnIndexes = new ArrayList<Integer>();
    private boolean[] continuousColumns;
    private String fileName;
    private String valFold = "None";
    private Hashtable<double[], ArrayList<Neighbour>> distanceMatrix;
    private Hashtable<double[], Hashtable<double[], Double>> distanceLookup;
    //private ArrayList<double[][]> numFolds =
    //Anvonds for det grafiska gronsnittet
    protected int[] probOrder;
    protected int nrOfClasses;
    protected int nrOfProbColumns =0; //Only used when extracting rules using probabilities, otherwise the variable will be 0.

    public ArffTableModel(String fileName) {
        this.fileName = fileName;
        arlNumericTypes.add("real");
        arlNumericTypes.add("integer");
        arlNumericTypes.add("numeric");
        parseArffFile(readArffFile(fileName));
        initColumnTypes();
        foldColumnIndex = calcFoldColumnIndex();
        //if(isClassificationTask()){
        if (!orginalFoldSpec) {
            if(!continuousColumns[targetColumnIndex])//only stratify automatically if classification problem
                stratifyData(foldSeed);
            initFoldColumn(nrOfFolds);
        }
        /*}else{
        nrOfFolds=4;
        if(foldColumnIndex==-1)
        initTimeSeriesFoldColumn(nrOfFolds);
        }*/
        validate();
    }
/*
    public ArffTableModel(String file, boolean isProb) {
        super();
        arlNumericTypes.add("real");
        arlNumericTypes.add("integer");
        arlNumericTypes.add("numeric");

        //Init attributes
        //parseArffFile(readArffFile(fileName));
        Instance first = instances.firstInstance();
        for (int a = 0; a < first.numAttributes(); a++) {
            Attribute att = first.attribute(a);
            targetColumnIndex = first.classIndex();

            ArrayList<String> allowedValues = new ArrayList<String>();
            if (att.isNumeric()) {
                allowedValues.add("numeric");
            } else {
                Enumeration vals = att.enumerateValues();
                while (vals.hasMoreElements()) {
                    String val = (String) vals.nextElement();
                    allowedValues.add(val);
                    //allowedValues.add(Utils.quote((String) vals.nextElement()));
                }
            }
            this.addColumn(att.name(), allowedValues);

        }

        //Read Data
        for (int row = 0; row < instances.numInstances(); row++) {
            for (int col = 0; col < instances.numAttributes(); col++) {
                if (instances.instance(row).attribute(col).isNumeric()) {
                    this.setValueAt(instances.instance(row).value(col), row, col);
                } else {
                    this.setValueAt(instances.instance(row).stringValue(col), row, col);
                }
            }
        }

        initColumnTypes();
        foldColumnIndex = -1;
        initFoldColumn(1);
        validate();
    }
    //ANvonds for console korningar.
*/
    public ArffTableModel(String fileName, int nrOfFolds, int foldSeed) {
        this.fileName = fileName;
        this.nrOfFolds = nrOfFolds;
        this.foldSeed = foldSeed;
        this.fileName = fileName;
        arlNumericTypes.add("real");
        arlNumericTypes.add("integer");
        arlNumericTypes.add("numeric");
        parseArffFile(readArffFile(fileName));
        initColumnTypes();
        foldColumnIndex = calcFoldColumnIndex();
        //  if(isClassificationTask()){
        if (!orginalFoldSpec) {
            if(!continuousColumns[targetColumnIndex])//only stratify automatically if classification problem
               stratifyData(foldSeed);
            initFoldColumn(nrOfFolds);
        }
        /* }else{
        nrOfFolds=4;
        if(foldColumnIndex==-1)
        initTimeSeriesFoldColumn(nrOfFolds);
        }*/
        validate();

    }

    public ArffTableModel() {
    }

    public ArffTableModel(ArffTableModel atm) {        
    	arlAllowedValues = atm.arlAllowedValues;
        arlNumValues = atm.arlNumValues;
        arlCatValues = atm.arlCatValues;
        tableName = atm.tableName;
        missingValues = atm.missingValues;
        arlNumericTypes = atm.arlNumericTypes;
        hstFoldCounters = atm.hstFoldCounters;
        foldColumnIndex = atm.foldColumnIndex;
        targetColumnIndex = atm.targetColumnIndex;
        nrOfFolds = atm.nrOfFolds;
        foldSeed = atm.foldSeed;
        hasFoldSpecification = atm.hasFoldSpecification; 
        orginalFoldSpec = atm.orginalFoldSpec;
        columnNames = atm.columnNames;
        hstTrainFolds = atm.hstTrainFolds;
        hstValFolds = atm.hstValFolds;
        hstTestFolds = atm.hstTestFolds;
        hstTrainFoldMinValues = atm.hstTrainFoldMinValues;
        hstTrainFoldMaxValues = atm.hstTrainFoldMaxValues;
        hstFoldSplits = atm.hstFoldSplits;
        arlNumData = atm.arlNumData;
        arlNormNumData = atm.arlNormNumData;
        arlFileComments = atm.arlFileComments;
        arlCategoryColumnIndexes = atm.arlCategoryColumnIndexes;
        arlContinousColumnIndexes = atm.arlContinousColumnIndexes;
        continuousColumns = atm.continuousColumns;
        fileName = atm.fileName;
        valFold = atm.valFold;
        distanceMatrix = atm.distanceMatrix;
        distanceLookup = atm.distanceLookup;
        probOrder = atm.probOrder;
        nrOfClasses = atm.nrOfClasses;
        nrOfProbColumns = atm.nrOfProbColumns; 
	}
    
	public static void main(String[] args) {
        ArffTableModel am = new ArffTableModel("sick");
        am.createDistanceMatrix();

    }
    String currentFold = "";
    //Hashtable<double[]> currentTestFold = new Hashtable<double[]>();

    public TreeSet<Neighbour> getKNN(double[] instance, PredictionContainer neigbhours) {
        if (distanceLookup == null) {
            createDistanceLookup();
        }
        TreeSet<Neighbour> arlN = new TreeSet<Neighbour>();
        Hashtable<double[], Double> lookup = distanceLookup.get(instance);
        Iterator it = neigbhours.values().iterator();
        for (int i = 0; i < neigbhours.size(); i++) {
            double[] n = ((Prediction) it.next()).getInstance();
            if (instance != n) {
                arlN.add(new Neighbour(n, lookup.get(n), ""));
            }
        }
        return arlN;
    }

    public ArrayList<Neighbour> getKNN(double[] instance, int k, String fold) {

        if (distanceMatrix == null) {
            createDistanceMatrix();
        }

        ArrayList<Neighbour> tm = distanceMatrix.get(instance);
        //    return tm;//*/

        ArrayList<Neighbour> realNeighbours = new ArrayList<Neighbour>();
        for (int i = 0; i < tm.size() && realNeighbours.size() < k; i++) {
            Neighbour n = tm.get(i);
            if (!n.getFold().equalsIgnoreCase(fold) && instance != n.getInstance()) {
                realNeighbours.add(n);
            }
        }
        return realNeighbours;//*/
    }

    public void destroyDistanceMatrixAndLookup() {
        distanceMatrix = null;
        distanceLookup = null;
    }

    public void createDistanceMatrix() {
        if (arlNormNumData == null);
        createNormalizedDataset();
        System.out.print("Creating distance matrix: ");
        distanceMatrix = new Hashtable<double[], ArrayList<Neighbour>>();
        for (int i = 0; i < arlNormNumData.size(); i++) {
            double[] normInstance = arlNormNumData.get(i);
            ArrayList<Neighbour> arlN = new ArrayList<Neighbour>();

            double[] instance = arlNumData.get(i);

            distanceMatrix.put(instance, arlN);
            for (int j = 0; j < arlNormNumData.size(); j++) {
                double[] normNeighbour = arlNormNumData.get(j);
                double[] neighbour = arlNumData.get(j);
                if (i != j) {
                    double distance = calcDistance(normInstance, normNeighbour);
                    arlN.add(new Neighbour(neighbour, distance, getValueAt(j, foldColumnIndex).toString()));//ctm.put(neighbour,distance);
                }
            }
            Collections.sort(arlN);
            int possibleknns = (int) (arlN.size() * 0.02 + 50);
            while (arlN.size() > possibleknns) {
                arlN.remove(arlN.size() - 1);//*/
            }
        }
        System.out.println("Ready");
    }

    public void createDistanceLookup() {
        if (arlNormNumData == null);
        createNormalizedDataset();
        System.out.print("Creating Lookup matrix: ");
        distanceLookup = new Hashtable<double[], Hashtable<double[], Double>>();
        for (int i = 0; i < arlNormNumData.size(); i++) {
            double[] normInstance = arlNormNumData.get(i);
            Hashtable<double[], Double> arlN = new Hashtable<double[], Double>();
            double[] instance = arlNumData.get(i);
            distanceLookup.put(instance, arlN);
            for (int j = 0; j < arlNormNumData.size(); j++) {
                double[] normNeighbour = arlNormNumData.get(j);
                double[] neighbour = arlNumData.get(j);
                if (i != j) {
                    double distance = calcDistance(normInstance, normNeighbour);
                    arlN.put(neighbour, distance);
                }
            }
        }
        System.out.println("Ready");
    }

    private double[][] getGlobalMaxMin() {
        int min = 0, max = 1;
        double[][] mm = new double[2][arlNumData.get(0).length];
        Arrays.fill(mm[min], Double.MAX_VALUE);
        Arrays.fill(mm[max], Double.MIN_VALUE);
        for (double[] instance : arlNumData) {
            for (int i = 0; i < instance.length - 1; i++) {
                if (instance[i] < mm[min][i]) {
                    mm[min][i] = instance[i];
                }
                if (instance[i] > mm[max][i]) {
                    mm[max][i] = instance[i];
                }
            }
        }
        return mm;
    }

    private void createNormalizedDataset() {
        System.out.print("Nomaralizing data: ");
        double[][] mm = getGlobalMaxMin();
        int min = 0, max = 1;
        arlNormNumData = new ArrayList<double[]>();
        for (double[] instance : arlNumData) {
            double[] normInstance = new double[instance.length];
            for (int i = 0; i < instance.length - 1; i++) {
                normInstance[i] = (instance[i] - mm[min][i]) / (mm[max][i] - mm[min][i]);
            }
            normInstance[normInstance.length - 1] = instance[instance.length - 1];
            arlNormNumData.add(normInstance);
        }
        System.out.println("Ready");
    }

    private double calcDistance(double[] instance, double[] neighbour) {
        Double distance = 0.0;
        double in = 0, ne = 0;
        for (int i = 0; i < instance.length - 1; i++) {
            if (continuousColumns[i]) {
                distance += Math.pow(instance[i] - neighbour[i], 2);
            } else {
                if (instance[i] != neighbour[i]) {
                    distance += 1;
                }
            }
        }
        return Math.sqrt(distance);
    }

    protected void initColumnTypes() {
        continuousColumns = new boolean[arlCategoryColumnIndexes.size() + arlContinousColumnIndexes.size()];
        for (int i = 0; i < getContinuousColumns().length; i++) {
            continuousColumns[i] = arlContinousColumnIndexes.contains(i);
        }

    }

    public ArffTableModel clone() {
        ArffTableModel m = new ArffTableModel();
        m.setDataVector(this.getDataVector(), this.columnIdentifiers); //evetuellt deep copy (vid ondring av foldnigen
        m.setArlAllowedValues(arlAllowedValues);
        m.setArlCatValues(arlCatValues);
        m.setArlCategoryColumnIndexes(arlCategoryColumnIndexes);
        m.setArlContinousColumnIndexes(arlContinousColumnIndexes);
        m.setArlFileComments(arlFileComments);
        m.setArlNumData(getArlNumData());
        m.setArlNumValues(arlNumValues);
        m.setArlNumericTypes(arlNumericTypes);
        m.setColumnNames(getColumnNames());
        m.setFileName(fileName);
        m.setFoldColumnIndex(foldColumnIndex);
        m.setFoldSeed(foldSeed);
        m.setHasFoldSpecification(hasFoldSpecification);
        m.setHstFoldCounters(hstFoldCounters);
        m.setHstTestFolds(hstTestFolds);
        m.setHstTrainFoldMaxValues(hstTrainFoldMaxValues);
        m.setHstTrainFoldMinValues(hstTrainFoldMinValues);
        m.setHstTrainFolds(hstTrainFolds);
        m.setHstValFolds(hstValFolds);
        m.hstFoldSplits=hstFoldSplits;
        m.setMissingValues(missingValues);
        m.setNrOfFolds(getNrOfFolds());
        m.setTableName(tableName);
        m.setTargetColumnIndex(targetColumnIndex);
        m.setValFold(valFold);
        m.setContinousColumns(continuousColumns);
        return m;
    }

    private void setContinousColumns(boolean[] cols) {
        continuousColumns = cols;
    }

    public Vector getColumnIdentifiers() {
        return super.columnIdentifiers;
    }

    public ArrayList<String> getFolds() {
        return arlAllowedValues.get(foldColumnIndex);
    }

    public boolean isNumericType(ArrayList<String> allowed) {
        return arlNumericTypes.contains(allowed.get(0).toLowerCase());
    }

    public void validate() {
        hstTrainFolds = new Hashtable<String, double[][]>();
        hstValFolds = new Hashtable<String, double[][]>();
        hstTestFolds = new Hashtable<String, double[][]>();
        hstTrainFoldMinValues = new Hashtable<String, double[]>();
        hstTrainFoldMaxValues = new Hashtable<String, double[]>();
        hstFoldSplits = new Hashtable<String, ArrayList<ArrayList<Double>>>();

        foldColumnIndex = calcFoldColumnIndex();
        if (foldColumnIndex == -1) {
            initFoldColumn(getNrOfFolds());
        }
        targetColumnIndex = getColumnCount() - 2;
        initNumData();
    }

    public void stratifyData(int seed) {
        foldSeed = seed;
        if (seed == -1) {
            Collections.shuffle(this.getDataVector(), new Random(System.currentTimeMillis()));
        } else {
            Collections.shuffle(this.getDataVector(), new Random(foldSeed));
        }
        sortAllRowsBy(this, targetColumnIndex, false);
        this.initNumData();
    }

    private void initNumData() {
        arlNumData = new ArrayList<double[]>();
        for (int r = 0; r < getRowCount(); r++) {
            double[] row = new double[targetColumnIndex + 1]; //Do not copy FOLDcOLUMN
            for (int c = 0; c <= targetColumnIndex; c++) {
                ArrayList<String> allowed = arlAllowedValues.get(c);
                if (isNumericType(allowed)) {
                    row[c] = Double.parseDouble(getValueAt(r, c).toString());
                } else {
                    row[c] = allowed.indexOf(getValueAt(r, c));
                }
            }
            getArlNumData().add(row);
        }
    }

    public ArffTableModel getOriginalModel() {
        return new ArffTableModel(fileName);
    }

    //Should always be used to get a fold as lazy initzilation is used.
    public double[][] getTrainFold(String fold) {
        double[][] data = hstTrainFolds.get(fold);
        if (data == null) {
            initFold(fold);
            data = hstTrainFolds.get(fold);
        }
        return data;
    }
    private String getDataAsString(double[][] data){
        String s="";
        NumberFormat nf = NumberFormat.getInstance();
        nf.setMaximumFractionDigits(10);
        nf.setMaximumIntegerDigits(10);
        nf.setGroupingUsed(false);
            for(int i = 0; i < continuousColumns.length; i ++){
                s+=getColumnIdentifiers().get(i) + " ";
                if(getContinuousColumns()[i])
                    s+= "numerical ";
                else
                    s+= "symbolic ";
            }
            s = s.trim();
            s += "\n";

            for (int r = 0; r < data.length; r++) {
                for (int c = 0; c < data[0].length; c++) {
                         s +=  nf.format(data[r][c]) + " ";
                                       //s += data[r][c] + " ";
                }
                s=s.trim()+"\n";
            }
        return s;        
    }

    public int getColumnByName(String name){
        if(getColumnNames()==null)
            columnNames = getColumnNames();
        for (int i = 0; i < getColumnNames().length; i++) {
            if(getColumnNames()[i].equalsIgnoreCase(name))
                return i;
        }
        return -1;
    }

        public String getTrainFoldAsString(String fold) {
          double[][] data = hstTrainFolds.get(fold);
                if (data == null) {
                initFold(fold);
                data = hstTrainFolds.get(fold);
            }
            //System.out.println(fold+"\n"+getDataAsString(data));
          return fold+"\n"+getDataAsString(data);
        }

         public String getTestFoldAsString(String fold) {
          double[][] data = hstTestFolds.get(fold);
                if (data == null) {
                initFold(fold);
                data = hstTrainFolds.get(fold);
            }
          return fold+"\n"+getDataAsString(data);
        }
        
    


    public double[] getTrainMaxValues(String fold) {
        double[] data = hstTrainFoldMaxValues.get(fold);
        if (data == null) {
            initFold(fold);
            data = hstTrainFoldMaxValues.get(fold);
        }
        return data;
    }

    public double[] getTrainMinValues(String fold) {
        double[] data = hstTrainFoldMinValues.get(fold);
        if (data == null) {
            initFold(fold);
            data = hstTrainFoldMinValues.get(fold);
        }
        return data;
    }

    public double[][] getTestFold(String fold) {
        double[][] data = hstTestFolds.get(fold);
        if (data == null) {
            initFold(fold);
            data = hstTestFolds.get(fold);
        }
        return data;
    }

    public double[][] getValFold(String fold) {
        double[][] data = hstValFolds.get(fold);
        if (data == null) {
            initFold(fold);
            data = hstValFolds.get(fold);
        }
        if (data == null) {
            data = new double[0][getColumnCount() - 1];
        }
        return data;
    }

    public void initFold(String fold) {
        ArrayList<double[]> trains = new ArrayList<double[]>();
        ArrayList<double[]> tests = new ArrayList<double[]>();
        ArrayList<double[]> vals = new ArrayList<double[]>();
        double[] min = new double[targetColumnIndex + 1];
        double[] max = new double[targetColumnIndex + 1];
        Arrays.fill(min, Double.MAX_VALUE);
        Arrays.fill(max, Double.MIN_VALUE);
        //      Collections.shuffle(getArlNumData());

        for (int r = 0; r < getArlNumData().size(); r++) {
           
            //int r = 0; r < arlNumData.size(); r++){
            double[] row = getArlNumData().get(r);
            if (getValueAt(r, foldColumnIndex).toString().equalsIgnoreCase(fold)) {
                tests.add(row);
            } else if (getValueAt(r, foldColumnIndex).toString().equalsIgnoreCase(valFold)) {
                vals.add(row);
            } else {//Train data
                for (int c = 0; c < row.length; c++) {
                    if (row[c] > max[c]) {
                        max[c] = row[c];
                    }
                    if (row[c] < min[c]) {
                        min[c] = row[c];
                    }
                }
                trains.add(row);
            }
        }


        double[][] tr = new double[trains.size()][getColumnCount() - 1];
        for (int r = 0; r < trains.size(); r++) {
            tr[r] = trains.get(r);// Arrays.copyOfRange(trains.get(r), 0, getColumnCount() - 1);
        }
        double[][] ts = new double[tests.size()][getColumnCount() - 1];
        for (int r = 0; r < tests.size(); r++) {
            ts[r] = tests.get(r);//Arrays.copyOfRange(tests.get(r), 0, getColumnCount() - 1);
        }
        double[][] vl = new double[vals.size()][getColumnCount() - 1];
        for (int r = 0; r < vals.size(); r++) {
            vl[r] = vals.get(r);//Arrays.copyOfRange(vals.get(r), 0, getColumnCount() - 1);
        }

        hstTrainFolds.put(fold, tr);
        hstTestFolds.put(fold, ts);
        hstValFolds.put(fold, vl);
        hstTrainFoldMaxValues.put(fold, max);
        hstTrainFoldMinValues.put(fold, min);
        hstFoldSplits.put(fold,getColumnSplits(fold));
    }

    public double getNumberOfSplits(int folds) {
        int nrOfsplits = 0;
        String fold = "";
        for (int i = 1; i <= folds; i++) {
            if (folds == 1) {
                fold = "All Data";
            } else {
                fold = "F" + i;
            }
            initFold(fold);
            ArrayList<ArrayList<Double>> splits = hstFoldSplits.get(fold);
            for (ArrayList<Double> column : splits) {
                nrOfsplits += column.size();
            }
        }
        return ((double) nrOfsplits) / folds;
    }
 /* public ArrayList<ArrayList<Double>> getColumnSplits(String fold){
        double[][] train = hstTrainFolds.get(fold);
        List<double[]> trainData = Arrays.asList(train);
        if(trainData == null){
            initFold(fold);
            train = hstTrainFolds.get(fold);
            trainData = Arrays.asList(train);
        }
        double[] firstRow = trainData.get(0);
        int columns = firstRow.length;
        int target = columns -1;
        double prevTarget;
        double prevValue,prevPrevValue;
        double lastSplit=Double.MIN_VALUE;
        boolean pure=true;
        double nrOfSplits=0;
        ArrayList<ArrayList<Double>> foldSplits = new ArrayList<ArrayList<Double>>();
        for(int c = 0; c < columns-1;c++){

            ArrayList<Double> columnSplits = new ArrayList<Double>();
            sortAllRowsBy(trainData, c, true);
            firstRow = trainData.get(0);
            prevTarget = firstRow[target];
            prevPrevValue = prevValue = firstRow[c];
            for(double[] d:trainData){
                            if(c==3 && d[3]==1.1)
                                System.out.println("nu");
                if(d[target]!=prevTarget){//om vi har en ny klass
                    if(d[c]!=prevValue){//om det akutella värdet inet är samma som det tidgare (vi kan göra en ren splits)
                        lastSplit = prevValue;// (prevValue+d[c])/2;
                        columnSplits.add(lastSplit);
                        prevTarget=d[target];
                    }else{//Unpure
                        pure=false;                       
                        if(prevPrevValue!= lastSplit){
                            columnSplits.add(prevPrevValue);
                            lastSplit = prevPrevValue;
                        }
                        prevTarget=-1;
                    }
                    
                }


                if(prevValue!=d[c]){//Om vi är i ett orent split värde och hittar ett nytt split värde skapa ett nytt split
                    if(!pure){
                        double newSplit = d[c]; //prevValue;//(prevValue+d[c])/2;
                        if(newSplit!= lastSplit){
                            columnSplits.add(newSplit);
                            lastSplit = newSplit;
                        }
                        pure=true;
                    }
                    prevPrevValue = prevValue;
                    prevValue = d[c];                    
                }
            }
            nrOfSplits += columnSplits.size();
        //    System.out.println(columnSplits.size());
            foldSplits.add(columnSplits);
          /*  for(Double d:columnSplits)
                System.out.println(d);
            System.out.println("");
        }
        return foldSplits;
    }//*/
public ArrayList<ArrayList<Double>> getColumnSplits(String fold){
        double[][] train = hstTrainFolds.get(fold).clone();
        List<double[]> trainData = Arrays.asList(train);
        if(trainData == null){
            initFold(fold);
            train = hstTrainFolds.get(fold);
            trainData = Arrays.asList(train.clone());
        }
        double[] firstRow = trainData.get(0);
        int columns = firstRow.length;
        ArrayList<ArrayList<Double>> foldSplits = new ArrayList<ArrayList<Double>>();
        for(int c = 0; c < columns-1;c++){

            ArrayList<Double> columnSplits = new ArrayList<Double>();
            sortAllRowsBy(trainData, c, true);
            for(double[] d:trainData){
                if(!columnSplits.contains(d[c]))
                    columnSplits.add(d[c]);
            }
            foldSplits.add(columnSplits);
        }
        return foldSplits;
    }//*/

    /*public ArrayList<ArrayList<Double>> getColumnSplits0(String fold){
        double[][] train = hstTrainFolds.get(fold);
        List<double[]> trainData = Arrays.asList(train);
        if(trainData == null){
            initFold(fold);
            train = hstTrainFolds.get(fold);
            trainData = Arrays.asList(train);
        }
        double[] firstRow = trainData.get(0);
        int columns = firstRow.length;
        int target = columns -1;
        double currentTarget;
        double lastPureSplit,currentValue,lastSplit=Double.MIN_VALUE;
        boolean pure=true;
        boolean existsPureSplit = false;
        double nrOfSplits=0;
        ArrayList<ArrayList<Double>> foldSplits = new ArrayList<ArrayList<Double>>();
        for(int c = 0; c < columns-1;c++){

            ArrayList<Double> columnSplits = new ArrayList<Double>();
            sortAllRowsBy(trainData, c, true);
            firstRow = trainData.get(0);
            currentTarget = firstRow[target];
            currentValue  = firstRow[c];
            lastPureSplit = Double.MIN_VALUE;
            existsPureSplit = false;
            pure = true;
            for(double[] d:trainData){
                if(d[c] != currentValue){
                    if(pure){
                        lastPureSplit = currentValue;
                        existsPureSplit = true;
                    }else{
                        if(existsPureSplit)
                            columnSplits.add(lastPureSplit);
                        if(lastPureSplit!=currentValue) //Undandtag för första splitten för att ej lägga till detta två gånger om det är orent
                            columnSplits.add(currentValue);
                        lastSplit = columnSplits.get(columnSplits.size()-1);
                        currentTarget = d[target];
                        pure = true;
                        existsPureSplit = false;
                    }
                    currentValue = d[c];
                }
                
                if(d[target]!=currentTarget){//om vi har en ny klass
                    pure=false;
                }
            }
            nrOfSplits += columnSplits.size();
        //    System.out.println(columnSplits.size());
            foldSplits.add(columnSplits);
          /*  for(Double d:columnSplits)
                System.out.println(d);
            System.out.println("");
        }
        return foldSplits;
    }//*/




    /*public ArrayList<ArrayList<Double>> getColumnSplits2(String fold){
        double[][] train = hstTrainFolds.get(fold);
        List<double[]> trainData = Arrays.asList(train);
        if(trainData == null){
            initFold(fold);
            train = hstTrainFolds.get(fold);
            trainData = Arrays.asList(train);
        }
        double[] firstRow = trainData.get(0);
        int columns = firstRow.length;
        int target = columns -1;
        double prevTarget;
        double prevValue,prevPrevValue;
        double lastSplit=Double.MIN_VALUE;
        boolean pure=true;
        double nrOfSplits=0;
        ArrayList<ArrayList<Double>> foldSplits = new ArrayList<ArrayList<Double>>();
        for(int c = 0; c < columns-1;c++){

            ArrayList<Double> columnSplits = new ArrayList<Double>();
            sortAllRowsBy(trainData, c, true);
            firstRow = trainData.get(0);
            prevTarget = firstRow[target];
            prevPrevValue = prevValue = firstRow[c];
            for(double[] d:trainData){
                            if(c==3 && d[3]==1.1)
                                System.out.println("nu");


                if(d[target]!=prevTarget){//om vi har en ny klass
                    if(d[c]!=prevValue){//om det akutella värdet inet är samma som det tidgare (vi kan göra en ren splits)
                        lastSplit = prevValue;// (prevValue+d[c])/2;
                        columnSplits.add(lastSplit);
                    }else{//Unpure
                        pure=false;
                        double newSplit = prevValue; //(prevValue+prevPrevValue)/2;
                        if(newSplit!= lastSplit){
                            columnSplits.add(newSplit);
                            lastSplit = newSplit;
                        }

                    }
                    prevTarget=d[target];
                }


                if(prevValue!=d[c]){//Om vi är i ett orent split värde och hittar ett nytt split värde skapa ett nytt split
                    if(!pure){
                        double newSplit = d[c]; //prevValue;//(prevValue+d[c])/2;
                        if(newSplit!= lastSplit){
                            columnSplits.add(newSplit);
                            lastSplit = newSplit;
                        }
                        pure=true;
                    }
                    prevPrevValue = prevValue;
                    prevValue = d[c];                    
                }
            }
            nrOfSplits += columnSplits.size();
        //    System.out.println(columnSplits.size());
            foldSplits.add(columnSplits);
          /*  for(Double d:columnSplits)
                System.out.println(d);
            System.out.println("");
        }
        return foldSplits;
    }//*/



    private void printNumData() {
        for (int r = 0; r < getArlNumData().size(); r++) {
            System.out.println("" + getArlNumData().get(r));
        }
    }

    public void printFold(String fold) {
        System.out.println("Fold----------------------->");
        double[][] train = getTrainFold(fold);
        for (int r = 0; r < train.length; r++) {
            for (int c = 0; c < train[0].length; c++) {
                System.out.print(train[r][c] + ",");
            }
            System.out.println("");
        }
    }

    public void addColumn(String name, ArrayList<String> allowedValues) {

        if (!hasFoldSpecification) {
            if (name.equalsIgnoreCase(FOLD_COLUMN_NAME)) {
                foldColumnIndex = getColumnCount();
                hasFoldSpecification = true;
            }
            super.addColumn(name);
            this.arlAllowedValues.add(allowedValues);
            initValueCounters(allowedValues);
            missingValues = new Boolean[super.getColumnCount()];
            Arrays.fill(missingValues, false);
            if (isNumericType(allowedValues)) {
                arlContinousColumnIndexes.add(getColumnCount() - 1);
            } else if (!name.equalsIgnoreCase(FOLD_COLUMN_NAME)) {
                arlCategoryColumnIndexes.add(getColumnCount() - 1);
            }
        } else {
            throw new ArrayIndexOutOfBoundsException("Fold column must be the last column in the data set. As the fold column already is set you cant add new columns!");
        }
    }

    public void removeColumn(int index) {
        arlAllowedValues.remove(index);
        targetColumnIndex--;
        foldColumnIndex--;
        arlCategoryColumnIndexes = new ArrayList<Integer>();
        arlContinousColumnIndexes = new ArrayList<Integer>();
        for (int i = 0; i < arlAllowedValues.size(); i++) {
            if (isNumericType(arlAllowedValues.get(i))) {
                arlContinousColumnIndexes.add(i);
            } else if ((arlAllowedValues.size() - 1) != i)//Dont add fold column
            {
                arlCategoryColumnIndexes.add(i);
            }
        }

        initNumData();
    }

    private void initValueCounters(ArrayList<String> allowedValues) {
        Hashtable<String, Integer> h = new Hashtable<String, Integer>();
        if (!isNumericType(allowedValues)) {
            //Nominal Column
            for (String type : allowedValues) {
                h.put(type, 0);
            }
        }
        arlCatValues.add(h);
        arlNumValues.add(new ArrayList<Double>());
    }

    public String[] getColumnNames() {

        String[] names = new String[super.columnIdentifiers.size()-1];
        for (int i = 0; i < super.columnIdentifiers.size() - 1; i++) {
            names[i] = (String) super.columnIdentifiers.get(i);
        } //.toArray();
        columnNames = names;
        return columnNames;
    }

    public ArrayList<Integer> getContiousColumnIndexes() {
        return arlContinousColumnIndexes;
    }

    public ArrayList<Integer> getCategoryColumnIndexes() {
        return arlCategoryColumnIndexes;
    }

    public int getNrOfCategorialColumns() {
        if (isClassificationTask()) {
            return arlCategoryColumnIndexes.size() - 1;
        } else {
            return arlCategoryColumnIndexes.size();
        }
    }

    public int getNrOfContinousColumns() {
        if (isClassificationTask()) {
            return arlContinousColumnIndexes.size();
        } else {
            return arlContinousColumnIndexes.size() - 1;
        }
    }

    public boolean isClassificationTask() {
        return !isNumericType(arlAllowedValues.get(targetColumnIndex));
    }

    public int getNrOfTargetCategories() {
        return arlAllowedValues.get(targetColumnIndex).size();
    }

    public int getNumberOfSplits(int column,String fold){
        ArrayList<Double> splits = hstFoldSplits.get(fold).get(column);
        return splits.size();
    }

    public double getGeneNumericalValue(int column, double value, String fold) {
        
        try{
            
        if (isNumericType(arlAllowedValues.get(column))) {
            ArrayList<Double> splits = hstFoldSplits.get(fold).get(column);
            int nrOfSplits = splits.size();
            int splitIndex = (int)Math.floor(value*nrOfSplits);
            if(splitIndex>=nrOfSplits)
                System.out.println("nuu");
            return splits.get((int)Math.floor(splitIndex));//*/
            
           /* double[] min = hstTrainFoldMinValues.get(fold);
            double[] max = hstTrainFoldMaxValues.get(fold);
            return min[column] + (max[column] - min[column]) * value;//*/
        } else {
            return Math.floor(arlAllowedValues.get(column).size() * value);
        }
          }catch(Exception e){
              e.printStackTrace();
              System.out.println(" col" + column + " rnd" + value);
              return -1;

          }
    }
    double lastError=-1;

    public double calcInternalNumericalValue(int column, double realValue, String fold) {
        //if (isNumericType(arlAllowedValues.get(column))) {
            /*double[] min = hstTrainFoldMinValues.get(fold);
            double[] max = hstTrainFoldMaxValues.get(fold);
            return (realValue-min[column]) / (max[column] - min[column]);//*/

            ArrayList<Double> splits = hstFoldSplits.get(fold).get(column);
            for (int i = 0; i < splits.size(); i++) {
                if (splits.get(i) == realValue) {
                    return ((double) i) / splits.size();
                }
                if (i < splits.size() - 1 && splits.get(i) < realValue && splits.get(i + 1) > realValue) {
                    splits.add(i + 1, realValue);
                    System.out.println("Added split: " + realValue + "for column " + column);
                    return ((double) (i + 1)) / splits.size();
                }
            }

            if (splits.get(0) > realValue) {
                splits.add(0, realValue);
                System.out.println("Added split: " + realValue + "for column " + column);
                return 0;
            }

            if (splits.get(splits.size() - 1) < realValue) {
                splits.add(realValue);
                System.out.println("Added split: " + realValue + "for column " + column);
                return ((double) (splits.size() - 1)) / splits.size();
            }
            System.out.println("Errroooooooooooooor cannot find or add J48 SPlits line 1005 in ArrfTableModel");

            //J48 uses splits inbetween and at instance. If we cant find a split choose the lower value of the closets split
            for (int i = 1; i < splits.size(); i++) {
                if (realValue > splits.get(i - 1) && realValue < splits.get(i)) {
                    return (double) (i - 1) / splits.size();
                }
            }
            return ((double) (splits.size() - 1)) / splits.size();
            //*/  } else {
            //   return -1; //Math.floor(arlAllowedValues.get(column).size() * rnd);
        //}
    }

    public double calcInternalCategoryValue(int catColumn, String category) {
        if (!isNumericType(arlAllowedValues.get(catColumn))) {
            ArrayList<String> values = arlAllowedValues.get(catColumn);
            double catIndex = values.indexOf(category);
            if(catIndex==-1)
                System.out.println("error: " + category + " not found for column: " + catColumn);

            return (1.0/values.size())*catIndex;
        } else {
            return -1; //Math.floor(arlAllowedValues.get(column).size() * rnd);
        }
    }

   /* public double getInternalValue(int column, double rnd, String fold) {
        double[][] trainData = hstTrainFolds.get(fold);
        int instance = (int) (rnd * trainData.length);
        return trainData[instance][column];
    }*/

    public double getRandomTargetValue(String fold) {
        return getGeneNumericalValue(targetColumnIndex, Options.rnd.nextDouble(), fold);
    }

    public String getRealCategerocialValue(int column, double internalValue) {
        String value="Error";
        ArrayList<String> col=null;
        try{
        col = arlAllowedValues.get(column);
        value = col.get((int)internalValue);
        }catch(Exception e){
            System.out.println("col:" + column + " iV:" + internalValue);
            e.printStackTrace();
        }
        return value;
    }

    /*public double getRealContinuousValue(int column, String fold, double internalValue) {
        double max = 0;
        double min = 0;
        if (hstTrainFoldMaxValues.get(fold) == null) {
            initFold(fold);
        }
        max = hstTrainFoldMaxValues.get(fold)[column];
        min = hstTrainFoldMinValues.get(fold)[column];
        return min + (max - min) * internalValue;
    }*/

    public int getTargetColumnIndex() {
        return targetColumnIndex;
    }

    public String getRealTargetClassValue(double classInternalValue) {
        return getRealCategerocialValue(targetColumnIndex, classInternalValue);
    }

    public int getRandomCategoricalInputColumn() {
        int index;

        int nrOfCats = getNrOfCategorialColumns();
        if (nrOfCats != 0) {
            index =  Options.rnd.nextInt(nrOfCats);
            return (int) arlCategoryColumnIndexes.get(index);
        }

        throw new ArrayIndexOutOfBoundsException("Categorical column was requiered but there are no categorical columns defined in the data set!");
    }

    public int getRandomContinuousInputColumn() throws GeneException {
        int index;

        int nrOfCons = getNrOfContinousColumns();
        if (nrOfCons != 0) {
            index = Options.rnd.nextInt(nrOfCons);
            return (int) arlContinousColumnIndexes.get(index);
        }
        throw new GeneException("Continous column was requiered but there are no contious columns defined in the data set!");
    }
    /*Used only when probabilities are predicted*/

    public int getRandomProbContinuousInputColumn() throws GeneException {
        int index;
        int nrOfCons = getNrOfContinousColumns() - getNrOfTargetCategories();
        if (nrOfCons != 0) {
            index = Options.rnd.nextInt(nrOfCons);
            return (int) arlContinousColumnIndexes.get(index);
        }
        throw new GeneException("Continous column was requiered but there are no contious columns defined in the data set!");
    }

    /*public int getRandomNonProbContinuousInputColumn() throws GeneException{
         int index;

     int nrOfCons = getNrOfContinousColumns();
     if(nrOfCons!=0){
         index = (int) Math.floor(Options.rnd.nextDouble() * (nrOfCons));
         return (int) arlContinousColumnIndexes.get(index);
     }
     throw new GeneException("Continous column was requiered but there are no contious columns defined in the data set!");
   }*/

    @Override
    public void setValueAt(Object o, int row, int column) {
        String value = o.toString();
        if (row == super.getRowCount()) {
            super.addRow(new Vector());
        }
        if (row > super.getRowCount()) {
            throw new ArrayIndexOutOfBoundsException("Row" + row + " does not exist yet, it is not possivle to add values at row > getRowCount.");
        }
        if (column >= super.getColumnCount()) {
            throw new ArrayIndexOutOfBoundsException("Column with index " + column + " does not exist");
        }
        if (value.charAt(0) == '?' || value.equals("NaN")) {
            missingValues[column] = true;
        } else {
            ArrayList<String> allowedColValues = arlAllowedValues.get(column);
            if (isNumericType(allowedColValues)) {
                arlNumValues.get(column).add(Double.parseDouble(value));
            } else {
                //Nominal column
                if (!getColumnName(column).equalsIgnoreCase(FOLD_COLUMN_NAME)) {
                    Hashtable<String, Integer> catVals = arlCatValues.get(column);
                    Integer typeCount = catVals.get(value);
                    if (typeCount != null) {
                        catVals.put(value, typeCount + 1);
                    } else {

                        throw new NumberFormatException("Value: " + value + " is not allowed for column " + super.getColumnName(column) + " only " + arlAllowedValues.get(column) + " are allowed");
                    }
                }
            }
        }
        super.setValueAt(value, row, column);
    }

    public String numToStr(int column, double value) {
        ArrayList<String> orgValues = arlAllowedValues.get(column);
        if (orgValues.size() > 1) {
            return "" + value;
        } else {
            return orgValues.get((int) value);
        }
    }

    public double strToNum(int column, String value) {
        ArrayList<String> orgValues = arlAllowedValues.get(column);
        if (orgValues.size() > 1) {
            return Double.parseDouble(value);
        } else {
            return orgValues.indexOf(value);
        }
    }

    public boolean containsFoldSpecification() {
        return hasFoldSpecification;
    }

    private int calcFoldColumnIndex() {
        for (int i = 0; i < super.getColumnCount(); i++) {
            if (super.getColumnName(i).equalsIgnoreCase(FOLD_COLUMN_NAME)) {
                return i;
            }
        }
        return -1;
    }

    public void initFoldColumn(int nrOfFolds) {

        Vector<String> column = new Vector<String>();
        ArrayList<String> allowed = new ArrayList<String>();
        for (int i = 1; i <= nrOfFolds; i++) {
            allowed.add("F" + i);
        }

        for (int i = 0; i < this.getRowCount(); i++) {
            if (foldColumnIndex == -1) {
                column.add(allowed.get(i % nrOfFolds));
            } else {
                super.setValueAt(allowed.get(i % nrOfFolds), i, foldColumnIndex);
            }
        }
        if (foldColumnIndex == -1) {
            super.addColumn(FOLD_COLUMN_NAME, column);

            foldColumnIndex = super.getColumnCount() - 1;
            hasFoldSpecification = true;
        } else {
            arlAllowedValues.remove(arlAllowedValues.size() - 1);
        }
        arlAllowedValues.add(allowed);
    }

    public void initTimeSeriesFoldColumn(int nrOfFolds) {
        Vector<String> column = new Vector<String>();
        ArrayList<String> allowed = new ArrayList<String>();
        allowed.add("TRAIN");
        allowed.add("TEST");

        for (int i = 0; i < this.getRowCount(); i++) {
            if (foldColumnIndex == -1) {
                column.add(allowed.get(i % nrOfFolds));
            } else {
                if (i < this.getRowCount() * (((double) nrOfFolds - 1) / nrOfFolds)) {
                    super.setValueAt(allowed.get(0), i, foldColumnIndex);
                } else {
                    super.setValueAt(allowed.get(1), i, foldColumnIndex);
                }
            }
        }
        if (foldColumnIndex == -1) {
            super.addColumn(FOLD_COLUMN_NAME, column);

            foldColumnIndex = super.getColumnCount() - 1;
            hasFoldSpecification = true;
        } else {
            arlAllowedValues.remove(arlAllowedValues.size() - 1);
        }
        arlAllowedValues.add(allowed);
    }

    public void initFoldColumn() {
        ArrayList<String> allowed = new ArrayList<String>();
        String s = "";
        for (int i = 0; i < this.getRowCount(); i++) {
            s = super.getValueAt(i, foldColumnIndex).toString();
            if (!allowed.contains(s)) {
                allowed.add(s);
            }
        }

        arlAllowedValues.remove(arlAllowedValues.size() - 1);
        arlAllowedValues.add(allowed);
    }

    public String getTableName() {
        return tableName;
    }

    public void setTableName(String tableName) {
        this.tableName = tableName;
    }

    private double calcColumnAvgValue(int column) {
        ArrayList<Double> values = arlNumValues.get(column);
        if (values.size() == 0) {
            return 0;
        }
        double sum = 0;
        for (double d : values) {
            sum += d;
        }
        return sum / values.size();
    }

    private String calcColumnTypeValue(int column) {
        Hashtable<String, Integer> values = arlCatValues.get(column);
        Set<String> keys = values.keySet();
        String type = "";
        int max = 0;
        for (String key : keys) {
            if (values.get(key) > max) {
                max = values.get(key);
                type = key;
            }
        }
        return type;
    }
// TOdo ensure that missing values are replaces when new instance is executed
    //Fix so that missing values are calculated for each fold instead of the whole dataset...no cheating!
    public void removeMissingValues() {
        for (int c = 0; c < missingValues.length; c++) {
            if (missingValues[c]) {
                if (arlAllowedValues.get(c).size() == 1) {
                    //numeric column
                    double avgValue = calcColumnAvgValue(c);
                    for (int r = 0; r < super.getRowCount(); r++) {
                        String value = super.getValueAt(r, c).toString();
                        if (value.charAt(0) == '?'|| value.equals("NaN")) {
                            super.setValueAt("" + avgValue, r, c);
                        }
                    }
                } else {
                    //Nominal column
                    String typeValue = calcColumnTypeValue(c);
                    for (int r = 0; r < super.getRowCount(); r++) {
                        String value = super.getValueAt(r, c).toString();
                        if (value.charAt(0) == '?'|| value.equals("NaN")) {
                            super.setValueAt(typeValue, r, c);
                        }
                    }
                }
            }
        }
    }

    public void save(String directory, String fileName) {
        PrintWriter p = null;
        String line = "";
        try {

            p = new PrintWriter(directory + fileName + ".txt");
            //    initFold(foldName);
            p.println("%Created with G-REX version 6.1 from the original file " + fileName + " instances are ordered in 10 folds starting with all instances in fold1 etc.");
            p.println("%The data has been translated to a numeric format. Missing values has been replaced with");
            p.println("%mean (continous) or type (nominal) values. The data is stratyfied with the foldseed " + foldSeed + ".");
            p.println("%Last column contains the target attribute.");
            p.println("%");
            p.println("%Below follows the orignal file comments (if any)");
            p.println("%");
            p.println("%");
            for (String s : arlFileComments) {
                p.println(s);
            }
            p.println("@Relation " + this.getTableName());
            Vector names = columnIdentifiers;
            ArrayList<String> vals;
            for (int c = 0; c < names.size() - 1; c++) {
                p.print("@Attribute " + names.get(c) + " ");
                vals = arlAllowedValues.get(c);
                if (vals.size() == 1) {
                    p.println(vals.get(0));
                } else {

                    String valsString = "{";
                    String translation = " %{";
                    for (int v = 0; v < vals.size() - 1; v++) {
                        valsString += (double) v + ",";
                        translation += (double) v + "=" + vals.get(v) + ",";
                    }
                    translation += vals.size() - 1.0 + "=" + vals.get(vals.size() - 1) + "}";
                    valsString += vals.size() - 1.0 + "}";
                    p.println(valsString + translation);
                }
            }



            ArrayList<double[]> data = new ArrayList<double[]>();
            ArrayList<String> folds = getFoldNames();
            double[][] foldData;
            for (String foldName : folds) {
                initFold(foldName);
                foldData = getTestFold(foldName);
                for (int i = 0; i < foldData.length; i++) {
                    data.add(foldData[i]);
                }

            }
            p.println("@Data");
            for (double[] d : data) {
                line = "";
                for (int c = 0; c < d.length; c++) {
                    line += d[c] + ",";
                }
                line = (String) line.subSequence(0, line.length() - 1);
//                    line += "0";//Train data
                p.println(line);
            }
            p.close();
        } catch (FileNotFoundException ex) {
            ErrorManager.getInstance().reportError(ex);
        } finally {
            p.close();
        }
    }

    public void saveFolds(String directory, String fileName) {
        //save(directory, fileName);
        // return;

        PrintWriter p = null;
        String line = "";
        try {
            ArrayList<String> folds = getFoldNames();
            for (String foldName : folds) {

                p = new PrintWriter(directory + fileName + foldName + ".txt");
                initFold(foldName);
                p.println("%Created with G-REX version 6.1 from the original file " + fileName + ". This is fold " + foldName + " of " + folds.size() + " folds.");
                p.println("%The data has been translated to a numeric format. Missing values has been replaced with");
                p.println("%mean (continous) or type (nominal) values. The data is stratyfied with the foldseed " + foldSeed + ".");
                p.println("%Last column contains column marker (0=Training data, 1=Test data). Second from last column");
                p.println("%contains the target attribute.");
                p.println("%");
                p.println("%Below follows the orignal file comments (if any)");
                p.println("%");
                p.println("%");
                for (String s : arlFileComments) {
                    p.println(s);
                }
                p.println("@Relation " + this.getTableName() + "_fold:" + foldName);
                Vector names = columnIdentifiers;
                ArrayList<String> vals;
                for (int c = 0; c < names.size() - 1; c++) {
                    p.print("@Attribute " + names.get(c) + " ");
                    vals = arlAllowedValues.get(c);
                    if (vals.size() == 1) {
                        p.println(vals.get(0));
                    } else {

                        String valsString = "{";
                        String translation = " %{";
                        for (int v = 0; v < vals.size() - 1; v++) {
                            valsString += (double) v + ",";
                            translation += (double) v + "=" + vals.get(v) + ",";
                        }
                        translation += vals.size() - 1.0 + "=" + vals.get(vals.size() - 1) + "}";
                        valsString += vals.size() - 1.0 + "}";
                        p.println(valsString + translation);
                    }
                }

                double[][] trainData = getTrainFold(foldName);
                double[][] valData = getValFold(foldName);
                double[][] testData = getTestFold(foldName);
                if (valData.length > 0) {
                    p.println("@Attribute " + FOLD_COLUMN_NAME + " {0,1,2} %{0=TRAIN,1=TEST(" + foldName + "),2=Val}");
                } else {
                    p.println("@Attribute " + FOLD_COLUMN_NAME + " {0,1} %{0=TRAIN,1=TEST(" + foldName + ")}");
                }
                p.println("@Data");
                for (int r = 0; r < trainData.length; r++) {
                    line = "";
                    for (int c = 0; c < trainData[0].length; c++) {
                        line += trainData[r][c] + ",";
                    }
                    //  line = (String) line.subSequence(0, line.length()-1);
                    line += "0";//Train data
                    p.println(line);
                }
                for (int r = 0; r < valData.length; r++) {
                    line = "";
                    for (int c = 0; c < trainData[0].length; c++) {
                        line += trainData[r][c] + ",";
                    }
                    line += "2";//Val data
                    //    line = (String) line.subSequence(0, line.length()-1);
                    p.println(line);
                }
                for (int r = 0; r < testData.length; r++) {
                    line = "";
                    for (int c = 0; c < testData[0].length; c++) {
                        line += testData[r][c] + ",";
                    }
                    line += "1";//Test
                    //line = (String) line.subSequence(0, line.length()-1);
                    p.println(line);
                }
                p.close();
            }


        } catch (FileNotFoundException ex) {
            ErrorManager.getInstance().reportError(ex);
        } finally {
            p.close();
        }//*/
    }

    public ArrayList<String> getFoldNames() {
        return arlAllowedValues.get(foldColumnIndex);
    }

    public int getFoldSeed() {
        return foldSeed;
    }

    public void fuzzit(int n) {
        fuzzyData(arlNumValues, n);
    }

    private void fuzzyData(ArrayList<ArrayList<Double>> newData, int n) {
        Fuzzyficator fuzz = new Fuzzyficator();
        String[] oldnames = this.getColumnNames();
        String[] fuzzyNames = null;


        String[] fuzzyNames2 = {"Low", "High"};
        String[] fuzzyNames3 = {"Low", "Medium", "High"};
        String[] fuzzyNames4 = {"Low", "MediumLow", "MediumHigh", "High"};
        String[] fuzzyNames5 = {"Low", "MediumLow", "Medium", "MediumHigh", "High"};

        if (n == 3) {
            fuzzyNames = fuzzyNames3;
        } else if (n == 2) {
            fuzzyNames = fuzzyNames2;
        } else if (n == 5) {
            fuzzyNames = fuzzyNames5;
        } else if (n == 4) {
            fuzzyNames = fuzzyNames4;
        } else {
            fuzzyNames = new String[n];
            for (int i = 0; i < n; i++) {
                fuzzyNames[i] = "" + 1.0 / i + "%";
            }
        }
        ArrayList<ArrayList<Double>> fuzzyData = fuzz.fuzzyfyData(arlNumData, getContinuousColumns(), n);
        ArrayList<ArrayList<String>> oldAllowedValues = (ArrayList<ArrayList<String>>) arlAllowedValues.clone();
        arlAllowedValues = new ArrayList<ArrayList<String>>();
        ArrayList<String> foldCol = new ArrayList<String>();
        ArrayList<String> targetCol = new ArrayList<String>();
        foldCol = new ArrayList<String>();
        targetCol = new ArrayList<String>();
        //Copy fold column and target column
        for (int r = 0; r < dataVector.size(); r++) {
            foldCol.add((String) ((Vector) dataVector.get(r)).get(foldColumnIndex));
            targetCol.add((String) ((Vector) dataVector.get(r)).get(targetColumnIndex));
        }

        //Remove all columns
        super.setColumnCount(0);
        //Add fuzzy columns oldname+ Fuzzy name
        int fuzzyColumnIndex = 0;
        for (int c = 0; c < oldnames.length - 1; c++) {
            if (continuousColumns[c]) {
                for (int fName = 0; fName < fuzzyNames.length; fName++) {
                    super.addColumn(fuzzyNames[fName] + "(" + oldnames[c] + ")", fuzzyData.get(fuzzyColumnIndex).toArray());
                    arlAllowedValues.add(oldAllowedValues.get(c));
                    fuzzyColumnIndex++;
                }
            } else {
                super.addColumn(oldnames[c], fuzzyData.get(fuzzyColumnIndex).toArray());
                arlAllowedValues.add(oldAllowedValues.get(c));
                fuzzyColumnIndex++;
            }
        }
        arlAllowedValues.add(oldAllowedValues.get(oldAllowedValues.size() - 2));
        arlAllowedValues.add(oldAllowedValues.get(oldAllowedValues.size() - 1));
        arlContinousColumnIndexes = new ArrayList<Integer>();
        arlCategoryColumnIndexes = new ArrayList<Integer>();
        super.addColumn(oldnames[oldnames.length - 1], targetCol.toArray());
        for (int col = 0; col < arlAllowedValues.size() - 1; col++) {
            if (isNumericType(arlAllowedValues.get(col))) {
                arlContinousColumnIndexes.add(col);
            } else {
                arlCategoryColumnIndexes.add(col);
            }
        }


        super.addColumn(FOLD_COLUMN_NAME, foldCol.toArray());
        foldColumnIndex = super.getColumnCount() - 1;
        targetColumnIndex = super.getColumnCount() - 2;
        validate();

    }

    public void aaddColumn(String name, ArrayList<String> allowedValues) {

        if (!hasFoldSpecification) {
            if (name.equalsIgnoreCase(FOLD_COLUMN_NAME)) {
                foldColumnIndex = getColumnCount();
                hasFoldSpecification = true;
            }
            super.addColumn(name);
            this.arlAllowedValues.add(allowedValues);
            initValueCounters(allowedValues);
            missingValues = new Boolean[super.getColumnCount()];
            Arrays.fill(missingValues, false);
            if (isNumericType(allowedValues)) {
                arlContinousColumnIndexes.add(getColumnCount() - 1);
            } else if (!name.equalsIgnoreCase(FOLD_COLUMN_NAME)) {
                arlCategoryColumnIndexes.add(getColumnCount() - 1);
            }
        } else {
            throw new ArrayIndexOutOfBoundsException("Fold column must be the last column in the data set. As the fold column already is set you cant add new columns!");
        }
    }

    public void rremoveColumn(int index) {
        arlAllowedValues.remove(index);
        targetColumnIndex--;
        foldColumnIndex--;
        arlCategoryColumnIndexes = new ArrayList<Integer>();
        arlContinousColumnIndexes = new ArrayList<Integer>();
        for (int i = 0; i < arlAllowedValues.size(); i++) {
            if (isNumericType(arlAllowedValues.get(i))) {
                arlContinousColumnIndexes.add(i);
            } else if ((arlAllowedValues.size() - 1) != i)//Dont add fold column
            {
                arlCategoryColumnIndexes.add(i);
            }
        }

        initNumData();
    }

     void parseArffFile(ArrayList<String> arffData) {
        //ArffTableModel table = new ArffTableModel();
        int row = 0;
        for (String line : arffData) {

            if (line.charAt(0) == '@') {
                //Reading definition
                StringTokenizer stk = new StringTokenizer(line, " \t");
                String token = stk.nextToken();
                if (token.equalsIgnoreCase("@attribute")) {
                    token = stk.nextToken();
                    String columnName = "";
                    String type = "";
                    if (token.charAt(0) == '\'') {
                        columnName = parseQuoteToken(token, stk);
                        while (stk.hasMoreTokens()) {
                            type += " " + stk.nextToken();
                        }
                    } else {
                        columnName = token;
                        while (stk.hasMoreTokens()) {
                            type += " " + stk.nextToken();
                        }
                        type = type.trim();
                    }
                    this.addColumn(columnName, parseArffColumnType(type));
                } else if (token.equalsIgnoreCase("@relation")) {
                    String name = "";
                    while (stk.hasMoreTokens()) {
                        name += " " + stk.nextToken();
                    }
                    name = name.trim();
                    name = name.replace("'", "");
                    this.setTableName(name);
                }
            } else {
                //Reading data line
                StringTokenizer stk = new StringTokenizer(line, ",;\t");
                String token = "";
                for (int c = 0; stk.hasMoreTokens(); c++) {
                    token = stk.nextToken();
                    if (token.charAt(0) == '\'') {
                        token = parseQuoteToken(token, stk);
                    }
                    this.setValueAt(token, row, c);
                }
                row++;
            }
        }
        removeMissingValues();
        orginalFoldSpec = calcFoldColumnIndex() != -1;
        this.validate();
    }

    private String parseQuoteToken(String startToken, StringTokenizer stk) {
        String token = startToken;
        while (!token.endsWith("'")) {
            token += " " + stk.nextToken();
        }
        return token.replace("'", "");
    }

    private ArrayList<String> parseArffColumnType(String type) {
        String values = type.replace("{", "").replace("}", "");
        StringTokenizer stk = new StringTokenizer(values, ",");

        ArrayList<String> allowedValues = new ArrayList<String>();
        String token = "";
        for (int i = 0; stk.hasMoreTokens(); i++) {
            token = stk.nextToken();
            token = token.trim();
            token = token.replace("'", "");
            allowedValues.add(token);
        }
        return allowedValues;
    }
    //Reads any file into an ArryList<String>,
    //First each line is trimmed of white spaces.
    //After the line is trimmed comments(everything after a %) is removed

    private ArrayList<String> readArffFile(String fileName) {
        ArrayList<String> tmpData = new ArrayList<String>();
        try {
            BufferedReader b;
            if (fileName.contains(File.separator)) {
                b = new BufferedReader(new FileReader(fileName));
            } else {
                b = new BufferedReader(new FileReader("." + File.separatorChar + "datasets" + File.separator + fileName));
            }
            //} else {
            //     b = new BufferedReader(new InputStreamReader(getClass().getResourceAsStream("/grex/Datasets/" + fileName + ".arff"))); //hypothyroid.arff")));
            // }
            String line = "";
            String[] result = null;
            String[] tmp;
            while ((line = b.readLine()) != null) {
                line = line.trim();
                if (!line.isEmpty() && line.charAt(0) != '%') {
                    result = line.split("%");
                    tmpData.add(result[0]);
                } else if (!line.isEmpty() && line.charAt(0) == '%') {
                    arlFileComments.add(line);
                }
            }
        } catch (FileNotFoundException ex) {
            ErrorManager.getInstance().reportError(ex);
            ex.printStackTrace();
        } catch (IOException ex) {
            ErrorManager.getInstance().reportError(ex);
            ex.printStackTrace();
        }
        return tmpData;
    }

    public ArrayList<String> getFileComments() {
        return arlFileComments;
    }

    public void sortAllRowsBy(Object data, int colIndex, boolean ascending) {
       
        if(data instanceof DefaultTableModel) {
            DefaultTableModel model = (DefaultTableModel) data;
            data = ((DefaultTableModel)model).getDataVector();
            Collections.sort(model.getDataVector(), new ColumnSorter(colIndex, ascending));
            model.fireTableStructureChanged();
        }else if(data instanceof List){
            Collections.sort((List) data, new ColumnSorter(colIndex, ascending));
        }
    }

    public String getValFold() {
        return valFold;
    }

    public void setValFold(String valFold) {
        this.valFold = valFold;
        validate();
    }

    public void setArlAllowedValues(
            ArrayList<ArrayList<String>> arlAllowedValues) {
        this.arlAllowedValues = arlAllowedValues;
    }

    public void setArlNumValues(ArrayList<ArrayList<Double>> arlNumValues) {
        this.arlNumValues = arlNumValues;
    }

    public void setArlNumericTypes(ArrayList<String> arlNumericTypes) {
        this.arlNumericTypes = arlNumericTypes;
    }

    public static void setFOLD_COLUMN_NAME(String aFOLD_COLUMN_NAME) {
        FOLD_COLUMN_NAME = aFOLD_COLUMN_NAME;
    }

    public void setArlCatValues(ArrayList<Hashtable<String, Integer>> arlCatValues) {
        this.arlCatValues = arlCatValues;
    }

    public void setMissingValues(Boolean[] missingValues) {
        this.missingValues = missingValues;
    }

    public void setHstFoldCounters(Hashtable<String, Integer> hstFoldCounters) {
        this.hstFoldCounters = hstFoldCounters;
    }

    public void setFoldColumnIndex(int foldColumnIndex) {
        this.foldColumnIndex = foldColumnIndex;
    }

    public void setTargetColumnIndex(int targetColumnIndex) {
        this.targetColumnIndex = targetColumnIndex;
    }

    public void setNrOfFolds(int nrOfFolds) {
        this.nrOfFolds = nrOfFolds;
    }

    public void setFoldSeed(int foldSeed) {
        this.foldSeed = foldSeed;
    }

    public void setHasFoldSpecification(boolean hasFoldSpecification) {
        this.hasFoldSpecification = hasFoldSpecification;
    }

    public void setColumnNames(String[] columnNames) {
        this.columnNames = columnNames;
    }

    public void setHstTrainFolds(Hashtable<String, double[][]> hstTrainFolds) {
        this.hstTrainFolds = hstTrainFolds;
    }

    public void setHstValFolds(Hashtable<String, double[][]> hstValFolds) {
        this.hstValFolds = hstValFolds;
    }

    public void setHstTestFolds(Hashtable<String, double[][]> hstTestFolds) {
        this.hstTestFolds = hstTestFolds;
    }

    public void setHstTrainFoldMinValues(Hashtable<String, double[]> hstTrainFoldMinValues) {
        this.hstTrainFoldMinValues = hstTrainFoldMinValues;
    }

    public void setHstTrainFoldMaxValues(Hashtable<String, double[]> hstTrainFoldMaxValues) {
        this.hstTrainFoldMaxValues = hstTrainFoldMaxValues;
    }

    public void setArlNumData(ArrayList<double[]> arlNumData) {
        this.arlNumData = arlNumData;
    }

    public void setArlFileComments(ArrayList<String> arlFileComments) {
        this.arlFileComments = arlFileComments;
    }

    public void setArlCategoryColumnIndexes(ArrayList<Integer> arlCategoryColumnIndexes) {
        this.arlCategoryColumnIndexes = arlCategoryColumnIndexes;
    }

    public void setArlContinousColumnIndexes(ArrayList<Integer> arlContinousColumnIndexes) {
        this.arlContinousColumnIndexes = arlContinousColumnIndexes;
    }

    public void setFileName(String fileName) {
        this.fileName = fileName;
    }

    public boolean[] getContinuousColumns() {
        return continuousColumns;
    }

    public ArrayList<double[]> getArlNumData() {
        return arlNumData;
    }

    public int getNrOfFolds() {
        return nrOfFolds;
    }
}
